#include <QStringList>
#include "PythonTreeBuilder.h"
#include "PythonGrammar30TreeConstants.h"
#include "SimpleNode.h"
#include "JJTState.h"
#include "PythonGrammar.h"

#include "ast/exprType.h"
#include "ast/Module.h"
#include "ast/Name.h"
#include "ast/Num.h"
#include "ast/Str.h"
#include "ast/For.h"
#include "ast/Pass.h"
#include "ast/Break.h"
#include "ast/Continue.h"
#include "ast/decoratorsType.h"
#include "ast/If.h"
#include "ast/AugAssign.h"
#include "ast/BinOp.h"
#include "ast/UnaryOp.h"
#include "ast/Import.h"
#include "ast/Attribute.h"
#include "ast/Starred.h"
#include "ast/modType.h"
#include "ast/Yield.h"
#include "ast/Expr.h"
#include "ast/Suite.h"
#include "ast/IdentityNode.h"
#include "ast/Decorators.h"
#include "ast/ComprehensionCollection.h"
#include "ast/ExtraArgValue.h"
#include "ast/ExtraArg.h"
#include "ast/DefaultArg.h"
#include "ast/FuncDefReturnAnn.h"
#include "ast/JfpDef.h"

PythonTreeBuilder::PythonTreeBuilder(JJTState* stack):
    _stack(stack),
    _lastOpened(NULL),
    _lastPop(NULL)
{

}

SimpleNodePtr PythonTreeBuilder::getLastOpened()
{
    return _lastOpened;
}

SimpleNodePtr PythonTreeBuilder::openNode(int id)
{
    SimpleNodePtr ret;

    switch (id){
        case JJTFILE_INPUT:         ret = ModulePtr(new Module(StmtTypeList())); break;
        case JJTFALSE:              ret = NamePtr(new Name("False", ExprContextType::Load, true, id)); break;
        case JJTTRUE :              ret = NamePtr(new Name("True", ExprContextType::Load, true, id)); break;
        case JJTNONE:               ret = NamePtr(new Name("None", ExprContextType::Load, true, id)); break;
        case JJTNAME:
        case JJTDOTTED_NAME:        ret = NamePtr(new Name("", ExprContextType::Load, false, id)); break;
        case JJTNUM:                ret = NumPtr(new Num(ObjectPtr(), -1, "")); break;
        case JJTSTRING:
        case JJTBINARY:             ret = StrPtr(new Str("", -1, false, false, false, id)); break;
        case JJTFOR_STMT:           ret = ForPtr(new For(exprTypePtr(), exprTypePtr(), StmtTypeList(), suiteTypePtr())); break;
        case JJTPASS_STMT:          ret = PassPtr(new Pass()); break;
        case JJTBREAK_STMT:         ret = BreakPtr(new Break()); break;
        case JJTCONTINUE_STMT:      ret = ContinuePtr(new Continue()); break;
        case JJTBEGIN_DECORATOR:    ret = decoratorsTypePtr(new decoratorsType(exprTypePtr(), ExprTypeList(), KeywordTypeList(), exprTypePtr(), exprTypePtr())); break;
        case JJTIF_STMT:            ret = IfPtr(new If(exprTypePtr(), StmtTypeList(), suiteTypePtr())); break;

        case JJTAUG_PLUS:           ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::Add, exprTypePtr(), id)); break;
        case JJTAUG_MINUS:          ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::Sub, exprTypePtr(), id)); break;
        case JJTAUG_MULTIPLY:       ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::Mult, exprTypePtr(), id)); break;
        case JJTAUG_DIVIDE:         ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::Div, exprTypePtr(), id)); break;
        case JJTAUG_MODULO:         ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::Mod, exprTypePtr(), id)); break;
        case JJTAUG_AND:            ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::BitAnd, exprTypePtr(), id)); break;
        case JJTAUG_OR:             ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::BitOr, exprTypePtr(), id)); break;
        case JJTAUG_XOR:            ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::BitXor, exprTypePtr(), id)); break;
        case JJTAUG_LSHIFT:         ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::LShift, exprTypePtr(), id)); break;
        case JJTAUG_RSHIFT:         ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::RShift, exprTypePtr(), id)); break;
        case JJTAUG_POWER:          ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::Pow, exprTypePtr(), id)); break;
        case JJTAUG_FLOORDIVIDE:    ret = AugAssignPtr(new AugAssign(exprTypePtr(), OperatorType::FloorDiv, exprTypePtr(), id)); break;

        case JJTOR_2OP:             ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::BitOr, exprTypePtr(), id)); break;
        case JJTXOR_2OP:            ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::BitXor, exprTypePtr(), id)); break;
        case JJTAND_2OP:            ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::BitAnd, exprTypePtr(), id)); break;
        case JJTLSHIFT_2OP:         ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::LShift, exprTypePtr(), id)); break;
        case JJTRSHIFT_2OP:         ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::RShift, exprTypePtr(), id)); break;
        case JJTADD_2OP:            ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::Add, exprTypePtr(), id)); break;
        case JJTSUB_2OP:            ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::Sub, exprTypePtr(), id)); break;
        case JJTMUL_2OP:            ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::Mult, exprTypePtr(), id)); break;
        case JJTDIV_2OP:            ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::Div, exprTypePtr(), id)); break;
        case JJTMOD_2OP:            ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::Mod, exprTypePtr(), id)); break;
        case JJTPOW_2OP:            ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::Pow, exprTypePtr(), id)); break;
        case JJTFLOORDIV_2OP:       ret = BinOpPtr(new BinOp(exprTypePtr(), OperatorType::FloorDiv, exprTypePtr(), id)); break;


        case JJTPOS_1OP:            ret = UnaryOpPtr(new UnaryOp(UnaryopType::UAdd, exprTypePtr(), id)); break;
        case JJTNEG_1OP:            ret = UnaryOpPtr(new UnaryOp(UnaryopType::USub, exprTypePtr(), id)); break;
        case JJTINVERT_1OP:         ret = UnaryOpPtr(new UnaryOp(UnaryopType::Invert, exprTypePtr(), id)); break;
        case JJTNOT_1OP:            ret = UnaryOpPtr(new UnaryOp(UnaryopType::Not, exprTypePtr(), id)); break;

        case JJTIMPORT:             ret = ImportPtr(new Import(AliasTypeList()));break;

        case JJTDOT_OP:             ret = AttributePtr(new Attribute(exprTypePtr(), NameTokPtr(), ExprContextType::Load, id));break;
        case JJTSTAR_EXPR:          ret = StarredPtr(new Starred(exprTypePtr(), ExprContextType::Store));break;

        default:                    ret = IdentityNodePtr(new IdentityNode(id)); break;
    }
    ret->setNodeId(id);
    _lastOpened = ret;
    return ret;
}

SimpleNodePtr PythonTreeBuilder::closeNode(const SimpleNodePtr& n, int arity)
{
    exprTypePtr value;
    suiteTypePtr orelseSuite;
    StmtTypeList body;
    exprTypePtr iter;
    exprTypePtr target;

    switch(n->nodeId()){
    case JJTFILE_INPUT:{
        ModulePtr m = n.staticCast<Module>();
        m->setBody(makeStmts(arity));
        return m;
    }

    case JJTFALSE:
    case JJTTRUE:
    case JJTNONE:
    case JJTNAME:
    case JJTNUM:
    case JJTPASS_STMT:
    case JJTBREAK_STMT:
    case JJTCONTINUE_STMT:
    case JJTSTRING:
    case JJTBINARY:
    case JJTBEGIN_DECORATOR:
    case JJTCOMMA:
    case JJTCOLON:
        return n;

    case JJTSUITE: {
        StmtTypeList stmts;
        for (int i = arity-1; i >= 0; i--) {
            SimpleNodePtr yield_or_stmt = _stack->popNode();
            if(yield_or_stmt.dynamicCast<Yield>()){
                stmts.append(ExprPtr(new Expr(yield_or_stmt.dynamicCast<Yield>())));
            } else {
                stmtTypePtr nod = yield_or_stmt.dynamicCast<stmtType>();
                if (nod) {
                    stmts.prepend(nod);
                } else {
                    QString msg = "ClassCastException: was: "+yield_or_stmt->type()+". Expected: stmtType";
                    //_stack->grammar()->addAndReport(e2, msg);
                    stmts.prepend(PassPtr(new Pass())); //recover from it with a valid node!
                }
            }
        }
        return SuitePtr(new Suite(stmts));
    }
    case JJTFOR_STMT: {
        orelseSuite.clear();
        if (_stack->nodeArity() == 5) {
            orelseSuite = popSuiteAndSuiteType();
        }

        body = popSuite();
        iter = _stack->popNode<exprType>();
        target = _stack->popNode<exprType>();
        _ctx.setStore(target);

        ForPtr forStmt = n.staticCast<For>();
        forStmt->setTarget(target);
        forStmt->setIter(iter);
        forStmt->setBody(body);
        forStmt->setOrelse(orelseSuite);
        return forStmt;
    }
    case JJTBEGIN_ELIF_STMT:
        return IfPtr(new If(exprTypePtr(), StmtTypeList(), suiteTypePtr(), JJTBEGIN_ELIF_STMT));

    case JJTIF_STMT:
        return handleIfConstruct(n, arity);

    case JJTDECORATORS: {
        DecoratorsTypeList list2;
        QList<SimpleNodePtr> listArgs;

        while(_stack->nodeArity() > 0){
            SimpleNodePtr node = _stack->popNode();
            while(!node.dynamicCast<decoratorsType>()) {
                if(node.dynamicCast<comprehensionType>()) {
                    listArgs.append(node);
                    listArgs.append(_stack->popNode()); //target
                } else if(node.dynamicCast<ComprehensionCollection>()) {
                    listArgs.append(node.dynamicCast<ComprehensionCollection>()->getGenerators().first());
                    listArgs.append(_stack->popNode()); //target
                } else {
                    listArgs.append(node);
                }
                node = _stack->popNode();
            }
            listArgs.append(node);//the decoratorsType
            list2.prepend(makeDecorator(listArgs).dynamicCast<decoratorsType>());
            listArgs.clear();
        }
        return DecoratorsPtr(new Decorators(list2, JJTDECORATORS));
    }
    case JJTSUBSCRIPTLIST:{
        SliceTypeList dims;
        for (int i = arity - 1; i >= 0; i--) {
            SimpleNodePtr sliceNode = _stack->popNode();
            if(sliceNode.dynamicCast<sliceType>()){
                dims.append(sliceNode.dynamicCast<sliceType>());
            } else if(sliceNode.dynamicCast<IdentityNode>()) {
                //this should be ignored...
                //this happens when parsing something like a[1,], whereas a[1,2] would not have this.
            } else {
                throw QString("Expected a sliceType or an IdentityNode. Received :"+sliceNode->type());
            }
        }
        return ExtSlicePtr(new ExtSlice(dims));
    }
    case JJTAUG_PLUS:
    case JJTAUG_MINUS:
    case JJTAUG_MULTIPLY:
    case JJTAUG_DIVIDE:
    case JJTAUG_MODULO:
    case JJTAUG_AND:
    case JJTAUG_OR:
    case JJTAUG_XOR:
    case JJTAUG_LSHIFT:
    case JJTAUG_RSHIFT:
    case JJTAUG_POWER:
    case JJTAUG_FLOORDIVIDE:{
        AugAssignPtr augAssign = n.dynamicCast<AugAssign>();
        exprTypePtr value1 = _stack->popNode<exprType>();
        exprTypePtr target1 = _stack->popNode<exprType>();
        _ctx.setAugStore(target1);
        augAssign->setTarget(target1);
        augAssign->setValue(value1);
        return n;
    }
    case JJTOR_BOOLEAN:
        return BoolOpPtr(new BoolOp(BoolopType::Or, makeExprs()));
    case JJTAND_BOOLEAN:
        return BoolOpPtr(new BoolOp(BoolopType::And, makeExprs()));
    case JJTCOMPARISION:{
        if(arity <= 2){
            throw QString("Internal error: To make a compare, at least 3 nodes are needed.");
        }
        int l = arity / 2;
        ExprTypeList comparators;
        IntList_ ops;
        ops.resize(l);
        for (int i = l-1; i >= 0; i--) {
            comparators.append(_stack->popNode<exprType>());
            SimpleNodePtr op = _stack->popNode();
            switch (op->nodeId()) {
                case JJTLESS_CMP:          ops[i] = CmpopType::Lt; break;
                case JJTGREATER_CMP:       ops[i] = CmpopType::Gt; break;
                case JJTEQUAL_CMP:         ops[i] = CmpopType::Eq; break;
                case JJTGREATER_EQUAL_CMP: ops[i] = CmpopType::GtE; break;
                case JJTLESS_EQUAL_CMP:    ops[i] = CmpopType::LtE; break;
                case JJTNOTEQUAL_CMP:      ops[i] = CmpopType::NotEq; break;
                case JJTIN_CMP:            ops[i] = CmpopType::In; break;
                case JJTNOT_IN_CMP:        ops[i] = CmpopType::NotIn; break;
                case JJTIS_NOT_CMP:        ops[i] = CmpopType::IsNot; break;
                case JJTIS_CMP:            ops[i] = CmpopType::Is; break;
            default:
                throw QString("Unknown cmp op:" + op->nodeId());
            }
        }
        return ComparePtr(new Compare(_stack->popNode<exprType>(), ops, comparators, JJTCOMPARISION));
    }
    case JJTLESS_CMP:
    case JJTGREATER_CMP:
    case JJTEQUAL_CMP:
    case JJTGREATER_EQUAL_CMP:
    case JJTLESS_EQUAL_CMP:
    case JJTNOTEQUAL_CMP:
    case JJTIN_CMP:
    case JJTNOT_IN_CMP:
    case JJTIS_NOT_CMP:
    case JJTIS_CMP:
        return n;

    case JJTOR_2OP:
    case JJTXOR_2OP:
    case JJTAND_2OP:
    case JJTLSHIFT_2OP:
    case JJTRSHIFT_2OP:
    case JJTADD_2OP:
    case JJTSUB_2OP:
    case JJTMUL_2OP:
    case JJTDIV_2OP:
    case JJTMOD_2OP:
    case JJTPOW_2OP:
    case JJTFLOORDIV_2OP:{
        BinOpPtr op = n.staticCast<BinOp>();
        exprTypePtr right = _stack->popNode<exprType>();
        exprTypePtr left = _stack->popNode<exprType>();
        op->setRight(right);
        op->setLeft(left);
        return n;
    }
    case JJTPOS_1OP:
    case JJTNEG_1OP:
    case JJTINVERT_1OP:
    case JJTNOT_1OP:
        n.dynamicCast<UnaryOp>()->setOperand(_stack->popNode<exprType>());
        return n;

    case JJTIMPORT:
        n.dynamicCast<Import>()->setNames(makeAliases(arity));
        return n;

    case JJTDOT_OP: {
        NameTokPtr attr = makeName(NameContextType::Attrib);
        value = _stack->popNode<exprType>();
        AttributePtr attribute = n.staticCast<Attribute>();
        attribute->setValue(value);
        attribute->setAttr(attr);
        return n;
    }
    case JJTBEGIN_DEL_STMT:
        return DeletePtr(new Delete(ExprTypeList()));

    case JJTDEL_STMT: {
        ExprTypeList exprs = makeExprs(arity-1);
        QList<SimpleNodePtr> nexprs;
        foreach(exprTypePtr node, exprs)
            nexprs.append(node);
        _ctx.setDelete(nexprs);
        DeletePtr d = _stack->popNode<Delete>();
        d->setTarget(exprs);
        return d;
    }

    case JJTDOTTED_NAME:{
        NamePtr name = n.dynamicCast<Name>();
        QStringList sb;
        for (int i = 0; i < arity; i++) {
            if (i > 0){
                sb.prepend(".");
            }
            NamePtr name0 = _stack->popNode<Name>();
            sb.prepend(name0->id());
            addSpecials(name0, name);
            //we have to set that, because if we later add things to the previous Name, we will now want it to be added to
            //the new name (comments will only appear later and may be added to the previous name -- so, we replace the previous
            //name specials list).
            name0->specialsBefore = name->getSpecialsBefore();
            name0->specialsAfter = name->getSpecialsAfter();
        }
        name->setId(sb.join(""));
        return name;
    }
    case JJTIMPORT_AS_NAME:
    case JJTDOTTED_AS_NAME:{
        NameTokPtr asname(NULL);
        if (arity > 1){
            asname = makeName(NameContextType::ImportName);
        }
        return aliasTypePtr(new aliasType(makeName(NameContextType::ImportName), asname));
    }

    case JJTSTAR_EXPR:{
        StarredPtr s = n.staticCast<Starred>();
        s->setValue(_stack->popNode<exprType>());
        _ctx.setStore(s);
        return s;
    }

    case JJTSTRJOIN: {
        StrPtr str2 = _stack->popNode<Str>();
        ObjectPtr o = _stack->popNode();
        StrJoinPtr ret;
        if(o.dynamicCast<Str>()){
            StrPtr str1 = o.staticCast<Str>();
            ret = StrJoinPtr(new StrJoin(ExprTypeList() << str1 << str2));
        } else {
            StrJoinPtr strJ = o.dynamicCast<StrJoin>();
            strJ->strs().append(str2);
            ret = strJ;
        }
        ret->beginLine = ret->strs().at(0)->beginLine;
        ret->beginColumn = ret->strs().at(0)->beginColumn;
        return ret;
    }
    }
    return onCloseNode(n, arity);
}

SimpleNodePtr PythonTreeBuilder::onCloseNode(const SimpleNodePtr& n, int arity)
{
    exprTypePtr value;
    ExprTypeList exprs;
    SuitePtr orelseSuite;
    StmtTypeList body;
    SuitePtr suite;
    int l;

    switch (n->nodeId()) {
    case JJTEXPR_STMT: {
        value = _stack->popNode<exprType>();
        if (arity > 1) {
            exprs = makeExprs(arity-1);
            QList<SimpleNodePtr> nexprs;
            foreach(exprTypePtr node, exprs)
                nexprs.append(node);
            _ctx.setStore(nexprs);
            return AssignPtr(new Assign(exprs, value));
        } else {
            return ExprPtr(new Expr(value));
        }
    }
    case JJTINDEX_OP:{
        sliceTypePtr slice = _stack->popNode<sliceType>();
        value = _stack->popNode<exprType>();
        return SubscriptPtr(new Subscript(value, slice, ExprContextType::Load));
    }
    case JJTBEGIN_FOR_ELSE_STMT:
        return SuitePtr(new Suite(StmtTypeList()));
    case JJTBEGIN_ELSE_STMT:
        return SuitePtr(new Suite(StmtTypeList()));
    case JJTBEGIN_WHILE_STMT:
        return WhilePtr(new While(exprTypePtr(), StmtTypeList(), suiteTypePtr()));
    case JJTWHILE_STMT: {
        orelseSuite.clear();;
        if (_stack->nodeArity() == 5){
            orelseSuite = popSuiteAndSuiteType();
        }

        body = popSuite();
        exprTypePtr test = _stack->popNode<exprType>();
        WhilePtr w = _stack->popNode<While>();
        w->setTest(test);
        w->setBody(body);
        w->setOrelse(orelseSuite.dynamicCast<suiteType>());
        return w;
    }
    case JJTCALL_OP:{
        exprTypePtr starargs;
        exprTypePtr kwargs;


        ExprTypeList args;
        KeywordTypeList keywords;

        for (int i = arity - 2; i >= 0; i--) {
            SimpleNodePtr node = _stack->popNode();
            if (node.dynamicCast<keywordType>()) {
                keywordTypePtr keyword = node.dynamicCast<keywordType>();
                keywords.prepend(keyword);
                if(!starargs){
                    keyword->setAfterstarargs(true); //note that we get things backward in the stack
                }
            } else if(node->nodeId() == JJTEXTRAARGVALUELIST) {
                ExtraArgValuePtr nstarargs = node.dynamicCast<ExtraArgValue>();
                starargs = nstarargs->value;
                addSpecialsAndClearOriginal(nstarargs, starargs);
            } else if (node->nodeId() == JJTEXTRAKEYWORDVALUELIST) {
                ExtraArgValuePtr nkwargs = node.dynamicCast<ExtraArgValue>();
                kwargs = nkwargs->value;
                addSpecialsAndClearOriginal(nkwargs, kwargs);
            } else if(node.dynamicCast<ComprehensionCollection>()) {
                //what can happen is something like print sum(x for x in y), where we have already passed x in the args, and then get 'for x in y'
                args.prepend(ListCompPtr(new ListComp(_stack->popNode<exprType>(), node.dynamicCast<ComprehensionCollection>()->getGenerators(), CompContextType::EmptyCtx)));
                i--; //popped node
            } else {
                args.prepend(node.dynamicCast<exprType>());
            }
        }

        exprTypePtr func = _stack->popNode<exprType>();
        CallPtr c = CallPtr(new Call(func, args, keywords, starargs, kwargs));
        addSpecialsAndClearOriginal(n, c);
        return c;
    }
    case JJTFUNCDEF_RETURN_ANNOTTATION:
        return FuncDefReturnAnnPtr(new FuncDefReturnAnn(_stack->popNode()));
    case JJTFUNCDEF: {
        suite = _stack->popNode<Suite>();
        body = suite->body();
        arity--;

        SimpleNodePtr funcDefReturnAnn = _stack->peekNode();
        exprTypePtr actualReturnAnnotation;
        if (funcDefReturnAnn.dynamicCast<FuncDefReturnAnn>()){
            _stack->popNode();
            actualReturnAnnotation = (funcDefReturnAnn.staticCast<FuncDefReturnAnn>()->node).staticCast<exprType>();
            arity--;
            addSpecialsAndClearOriginal(funcDefReturnAnn, actualReturnAnnotation);
        }
        argumentsTypePtr arguments = makeArguments(arity-1);
        NameTokPtr nameTok = makeName(NameContextType::FunctionName);
        //decorator is always null at this point... it's decorated later on
        FunctionDefPtr funcDef = FunctionDefPtr(new FunctionDef(nameTok, arguments, body, DecoratorsTypeList(), actualReturnAnnotation));
        addSpecialsAndClearOriginal(suite, funcDef);
        setParentForFuncOrClass(body, funcDef);
        return funcDef;
    }
    case JJTTFPDEF:{
        NamePtr tfpdefName;
        exprTypePtr typeDef;
        if (arity == 1) {
            tfpdefName = _stack->popNode<Name>();
        } else if (arity == 2) {
            typeDef = _stack->popNode<exprType>();
            tfpdefName = _stack->popNode<Name>();
        } else {
            throw QString("Unexpected arity: %1").arg(arity);
        }
        return JfpDefPtr(new JfpDef(tfpdefName, typeDef));
    }
    case JJTONLYKEYWORDARG2:
    case JJTDEFAULTARG2: {
        DefaultArgPtr defaultArg;
        JfpDefPtr jfpDef;
        if (arity == 1) {
            jfpDef = _stack->popNode<JfpDef>();
            defaultArg = DefaultArgPtr(new DefaultArg(jfpDef->nameNode, exprTypePtr(), jfpDef->typeDef, n->nodeId()));
        } else if(arity == 2) {
            exprTypePtr defaultValue = _stack->popNode<exprType>();
            jfpDef = _stack->popNode<JfpDef>();
            defaultArg = DefaultArgPtr(new DefaultArg(jfpDef->nameNode, defaultValue, jfpDef->typeDef, n->nodeId()));
        } else {
            throw QString("Unexpected arity: %1").arg(arity);
        }
        return defaultArg;
    }
    case JJTONLYKEYWORDARG:
    case JJTDEFAULTARG: {
        //no type definition in this case
        if(arity == 1){
            return DefaultArgPtr(new DefaultArg(_stack->popNode<exprType>(), exprTypePtr(), exprTypePtr(), n->nodeId()));
        }
        exprTypePtr parameter = _stack->popNode<exprType>();
        return DefaultArgPtr(new DefaultArg(_stack->popNode<exprType>(), parameter, exprTypePtr(), n->nodeId()));
    }
    case JJTEXTRAARGLIST: {
        if(arity == 0){
            //nothing here (just '*')
            return ExtraArgPtr(new ExtraArg(NameTokPtr(), JJTEXTRAARGLIST, exprTypePtr()));
        }
        return ExtraArgPtr(new ExtraArg(makeName(NameContextType::VarArg), JJTEXTRAARGLIST));
    }
    case JJTEXTRAKEYWORDLIST:
        return ExtraArgPtr(new ExtraArg(makeName(NameContextType::KwArg), JJTEXTRAKEYWORDLIST));
    case JJTEXTRAARGLIST2: {//with type declaration
        if(arity == 0){
            //nothing here (just '*')
            return ExtraArgPtr(new ExtraArg(NameTokPtr(), JJTEXTRAARGLIST, exprTypePtr()));
        }
        JfpDefPtr jfpDef = _stack->popNode<JfpDef>();
        NameTokPtr jfpDefName = makeName(NameContextType::VarArg, jfpDef->nameNode);
        ExtraArgPtr extra = ExtraArgPtr(new ExtraArg(jfpDefName, JJTEXTRAARGLIST, jfpDef->typeDef));
        return extra;
    }
    case JJTEXTRAKEYWORDLIST2: {//with type declaration
        JfpDefPtr jfpDef = _stack->popNode<JfpDef>();
        return ExtraArgPtr(new ExtraArg(makeName(NameContextType::KwArg, jfpDef->nameNode), JJTEXTRAKEYWORDLIST, jfpDef->typeDef));
    }
    case JJTDECORATED: {
        if(_stack->nodeArity() != 2){
            throw QString("Expected 2 nodes at this context, found: %1").arg(arity);
        }
        SimpleNodePtr def = _stack->popNode();
        DecoratorsPtr decorators = _stack->popNode<Decorators>();
        if (def.dynamicCast<ClassDef>()) {
            ClassDefPtr classDef = def.dynamicCast<ClassDef>();
            classDef->setDecs(*decorators->exp());
        } else {
            FunctionDefPtr fDef = def.dynamicCast<FunctionDef>();
            fDef->setDecs(*decorators->exp());
        }
        return def;
    }
    case JJTCLASSDEF: {
        suite = _stack->popNode<Suite>();
        body = suite->body();
        int nodeArity = _stack->nodeArity()-1;
        KeywordTypeList classDefKeywords;
        exprTypePtr starargs;
        exprTypePtr kwargs;

        int loopTo = nodeArity;
        for (int i = 0; i < loopTo; ++i) {
            SimpleNodePtr node = _stack->peekNode();
            if(node.dynamicCast<keywordType>()){
                _stack->popNode();
                keywordTypePtr keyword = node.dynamicCast<keywordType>();
                classDefKeywords.append(keyword);
                if(starargs){
                    keyword->setAfterstarargs(true); //note that we get things backward in the stack
                }
                nodeArity--;
            } else if(node.dynamicCast<ExtraArgValue>()) {
                if (node->nodeId() == JJTEXTRAARGVALUELIST) {
                    ExtraArgValuePtr nstarargs = _stack->popNode<ExtraArgValue>();
                    starargs = nstarargs->value;
                    addSpecialsAndClearOriginal(nstarargs, starargs);
                    nodeArity--;
                } else if (node->nodeId() == JJTEXTRAKEYWORDVALUELIST) {
                    ExtraArgValuePtr nkwargs = _stack->popNode<ExtraArgValue>();
                    kwargs = nkwargs->value;
                    addSpecialsAndClearOriginal(nkwargs, kwargs);
                    nodeArity--;
                }
            } else {
                break;
            }
        }
        if (classDefKeywords.length() > 1) {
            KeywordTypeList rclassDefKeywords;
            for(int i = classDefKeywords.length()-1; i == 0; --i)
                rclassDefKeywords.append(classDefKeywords[i]);
            classDefKeywords = rclassDefKeywords;
        }

        ExprTypeList bases = makeExprs(nodeArity);
        NameTokPtr nameTok = makeName(NameContextType::ClassName);
        //decorator is always null at this point... it's decorated later on
        ClassDefPtr classDef = ClassDefPtr(new ClassDef(nameTok, bases, body, DecoratorsTypeList(), classDefKeywords, starargs, kwargs));
        addSpecialsAndClearOriginal(suite, classDef);
        setParentForFuncOrClass(body, classDef);
        return classDef;
    }
    case JJTBEGIN_RETURN_STMT:
        return ReturnPtr(new Return(exprTypePtr()));
    case JJTRETURN_STMT: {
        if (arity == 2)
            value = _stack->popNode<exprType>();
        else
            value.clear();
        ReturnPtr ret = _stack->popNode<Return>();
        ret->setValue(value);
        return ret;
    }
    case JJTYIELD_STMT:
        return _stack->popNode();
    case JJTYIELD_EXPR: {
        exprTypePtr yieldExpr;
        if(arity > 0){
            //we may have an empty yield, so, we have to check it before
            yieldExpr = _stack->popNode<exprType>();
        }
        return YieldPtr(new Yield(yieldExpr));
    }
    case JJTRAISE_STMT: {
        exprTypePtr from;
        if (arity >= 2)
             from = _stack->popNode<exprType>();

        exprTypePtr type;
        if (arity >= 1)
            type = _stack->popNode<exprType>();
        return RaisePtr(new Raise(type, exprTypePtr(), exprTypePtr(), from));
    }
    case JJTGLOBAL_STMT:
        return GlobalPtr(new Global(makeIdentifiers(NameContextType::GlobalName), exprTypePtr()));
    case JJTNONLOCAL_STMT:
        return NonLocalPtr(new NonLocal(makeIdentifiers(NameContextType::NonLocalName), exprTypePtr()));
    case JJTASSERT_STMT: {
        exprTypePtr msg;
        if (arity == 2)
            msg = _stack->popNode<exprType>();
        exprTypePtr test = _stack->popNode<exprType>();
        return AssertPtr(new Assert(test, msg));
    }
    case JJTBEGIN_TRY_STMT:
        //we do that just to get the specials
        return TryExceptPtr(new TryExcept(StmtTypeList(), ExcepthandlerTypeList(), suiteTypePtr()));
    case JJTTRYELSE_STMT:
        orelseSuite = popSuiteAndSuiteType();
        return orelseSuite;
    case JJTTRYFINALLY_OUTER_STMT:
        orelseSuite = popSuiteAndSuiteType();
        return TryFinallyPtr(new TryFinally(StmtTypeList(), orelseSuite)); //it does not have a body at this time... it will be filled with the inner try..except
    case JJTTRY_STMT: {
        TryFinallyPtr outer;
        if (_stack->peekNode<TryFinally>()) {
            outer = _stack->popNode<TryFinally>();
            arity--;
        }
        orelseSuite.clear();
        if (_stack->peekNode<suiteType>()) {
            orelseSuite = _stack->popNode<Suite>();
            arity--;
        }

        l = arity ;
        ExcepthandlerTypeList handlers;
        for (int i = l - 1; i >= 0; i--) {
            handlers.append(_stack->popNode<excepthandlerType>());
        }
        suite = _stack->popNode<Suite>();
        TryExceptPtr tryExc = _stack->popNode<TryExcept>();
        if (outer != NULL) {
            outer->beginLine = tryExc->beginLine;
        }
        tryExc->setBody(suite->body());
        tryExc->setHandlers(handlers);
        tryExc->setOrelse(orelseSuite.dynamicCast<suiteType>());
        addSpecials(suite, tryExc);
        if (!outer) {
            return tryExc;
        } else {
            if(outer->body().length()){
                throw QString("Error. Expecting null body to be filled on try..except..finally");
            }
            outer->setBody(StmtTypeList(tryExc));
            return outer;
        }
    }
    case JJTBEGIN_TRY_ELSE_STMT:
        //we do that just to get the specials
        return SuitePtr(new Suite(StmtTypeList()));
    case JJTBEGIN_EXCEPT_CLAUSE:
        return excepthandlerTypePtr(new excepthandlerType(exprTypePtr(), exprTypePtr(), StmtTypeList()));
    case JJTEXCEPT_CLAUSE: {
        suite = _stack->popNode<Suite>();
        body = suite->body();
        exprTypePtr excname;
        if (arity == 4)
            excname = _stack->popNode<exprType>();
        if (excname){
            _ctx.setStore(excname);
        }
        exprTypePtr type;
        if (arity >= 3)
            type = _stack->popNode<exprType>();
        excepthandlerTypePtr handler = _stack->popNode<excepthandlerType>();
        handler->setType_(type);
        handler->setName(excname);
        handler->setBody(body);
        addSpecials(suite, handler);
        return handler;
    }
    case JJTBEGIN_FINALLY_STMT:
        //we do that just to get the specials
        return SuitePtr(new Suite(StmtTypeList()));
    case JJTTRYFINALLY_STMT: {
        SuitePtr finalBody = popSuiteAndSuiteType();
        body = popSuite();
        //We have a try..except in the stack, but we will change it for a try..finally
        //This is because we recognize a try..except in the 'try:' token, but actually end up with a try..finally
        TryExceptPtr tryExcept = _stack->popNode<TryExcept>();
        TryFinallyPtr tryFinally = TryFinallyPtr(new TryFinally(body, finalBody));
        tryFinally->beginLine = tryExcept->beginLine;
        tryFinally->beginColumn = tryExcept->beginColumn;
        addSpecialsAndClearOriginal(tryExcept, tryFinally);
        return tryFinally;
    }
    case JJTWITH_STMT:
        return makeWithStmt(arity);
    case JJTWITH_ITEM:
        return makeWithItem(arity);
    case JJTEXTRAKEYWORDVALUELIST:
        return ExtraArgValuePtr(new ExtraArgValue(_stack->popNode<exprType>(), JJTEXTRAKEYWORDVALUELIST));
    case JJTEXTRAARGVALUELIST:
        return ExtraArgValuePtr(new ExtraArgValue(_stack->popNode<exprType>(), JJTEXTRAARGVALUELIST));
    case JJTARGUMENT: {
        SimpleNodePtr keyword = _stack->popNode();
        if (keyword.dynamicCast<keywordType>()) {
            NameTokPtr nameTok = makeName(NameContextType::KeywordName);
            keyword.dynamicCast<keywordType>()->setArg(nameTok);
        }
        return keyword;
    }
    case JJTKEYWORD: {
        value = _stack->popNode<exprType>();
        return keywordTypePtr(new keywordType(NameTokPtr(), value, false));
    }
    case JJTTUPLE: {
        if (_stack->nodeArity() > 0) {
            SimpleNodePtr peeked = _stack->peekNode();
            if (peeked.dynamicCast<ComprehensionCollection>()) {
                ComprehensionCollectionPtr col = _stack->popNode<ComprehensionCollection>();
                return ListCompPtr(new ListComp(_stack->popNode<exprType>(), col->getGenerators(), CompContextType::TupleCtx));
            }
        }
        return makeTuple(n);
    }
    case JJTLIST: {
        if (_stack->nodeArity() > 0 && _stack->peekNode<ComprehensionCollection>()) {
            ComprehensionCollectionPtr col = _stack->popNode<ComprehensionCollection>();
            return ListCompPtr(new ListComp(_stack->popNode<exprType>(), col->getGenerators(), CompContextType::ListCtx));
        }
        return ListPtr(new List(makeExprs(), ExprContextType::Load));
    }
    case JJTSET:
        return SetPtr(new Set(ExprTypeList()));
    case JJTDICTIONARY:
        return makeDictionaryOrSet(arity);
    case JJTTEST:
        if(arity == 2){
            IfExpPtr node = _stack->popNode<IfExp>();
            node->setBody(_stack->popNode<exprType>());
            return node;
        }else{
            return _stack->popNode();
        }
    case JJTIF_EXP: {
        exprTypePtr ifExprOrelse = _stack->popNode<exprType>();
        exprTypePtr ifExprTest = _stack->popNode<exprType>();
        return IfExpPtr(new IfExp(ifExprTest, exprTypePtr(), ifExprOrelse));
    }
    case JJTLAMBDEF_NOCOND:
    case JJTLAMBDEF: {
        exprTypePtr test = _stack->popNode<exprType>();
        argumentsTypePtr arguments = makeArguments(arity - 1);
        return LambdaPtr(new Lambda(arguments, test));
    }
    case JJTELLIPSIS:
        return EllipsisPtr(new Ellipsis());

    case JJTELLIPSIS_AS_NAME:
        return NamePtr(new Name("...", ExprContextType::Load, true));

    case JJTSLICE: {
        QList<SimpleNodePtr> arr;
        for (int i = arity-1; i >= 0; i--) {
            arr.append(_stack->popNode());
        }

        QVector<exprTypePtr> values;
        values.resize(3);
        int k = 0;
        QList<ObjectPtr> specialsBefore;
        QList<ObjectPtr> specialsAfter;

        for (int j = 0; j < arity; j++) {
            if (arr[j]->nodeId() == JJTCOLON) {
                if (arr[j]->specialsBefore.length()) {
                    specialsBefore.append(arr[j]->specialsBefore);
                    arr[j]->specialsBefore.clear(); //this nodes may be reused among parses, so, we have to erase the specials
                }
                if (arr[j]->specialsAfter.length()) {
                    specialsAfter.append(arr[j]->specialsAfter);
                    arr[j]->specialsAfter.clear();
                }
                k++;
            } else {
                values[k] = arr[j].dynamicCast<exprType>();
                if (specialsBefore.length() > 0) {
                    values[k]->specialsBefore.append(specialsBefore);
                    specialsBefore.clear();
                }
                if (specialsAfter.length() > 0) {
                    values[k]->specialsBefore.append(specialsAfter);
                    specialsAfter.clear();
                }
            }
        }
        SimpleNodePtr sliceRet;
        if (k == 0) {
            sliceRet = IndexPtr(new Index(values[0]));
        } else {
            sliceRet = SlicePtr(new Slice(values[0], values[1], values[2]));
        }
        //this may happen if we have no values
        sliceRet->specialsBefore.append(specialsBefore);
        sliceRet->specialsAfter.append(specialsAfter);
        specialsBefore.clear();
        specialsAfter.clear();
        return sliceRet;
    }
    case JJTCOMP_FOR:
        return makeCompFor(arity);

    case JJTIMPORTFROM:
        return makeImportFrom25Onwards(arity);

    }

    return SimpleNodePtr(NULL);
}


StmtTypeList PythonTreeBuilder::makeStmts(int l)
{
    StmtTypeList stmts;
    for (int i = l - 1; i >= 0; i--) {
        //qDebug() << _stack->peekNode<stmtType>()->toString();
        stmts.prepend(_stack->popNode<stmtType>());
    }
    return stmts;
}

StmtTypeList PythonTreeBuilder::popSuite()
{
    return getBodyAndSpecials();
}

SuitePtr PythonTreeBuilder::popSuiteAndSuiteType()
{
    SuitePtr s = _stack->popNode<Suite>();
    SuitePtr orelseSuite = _stack->popNode<Suite>();
    orelseSuite->setBody(s->body());
    addSpecialsAndClearOriginal(s, orelseSuite);
    return orelseSuite;
}

void PythonTreeBuilder::addSpecialsAndClearOriginal(const SimpleNodePtr& from, const SimpleNodePtr& to)
{
    addSpecials(from, to);
    if (from->specialsBefore.length()){
        from->specialsBefore.clear();
    }
    if (from->specialsAfter.length()) {
        from->specialsAfter.clear();
    }
}

void PythonTreeBuilder::addSpecials(const SimpleNodePtr& from, const SimpleNodePtr& to)
{
    if (from->specialsBefore.length() > 0){
        to->specialsBefore.append(from->specialsBefore);
    }
    if (from->specialsAfter.length() > 0) {
        to->specialsAfter.append(from->specialsAfter);
    }
}

void PythonTreeBuilder::addSpecialsBefore(const SimpleNodePtr& from, const SimpleNodePtr& to)
{
    if (from->specialsBefore.length() > 0) {
        to->specialsBefore.append(from->specialsBefore);
    }
    if (from->specialsAfter.length() > 0) {
        to->specialsBefore.append(from->specialsAfter);
    }
}

StmtTypeList PythonTreeBuilder::getBodyAndSpecials()
{
    SuitePtr suite = _stack->popNode<Suite>();
    //qDebug() << suite->toString();
    if(!suite)
        return StmtTypeList();
    StmtTypeList body = suite->body();
    if(body.length() > 0){
        //Check size (this can happen when parsing wrong grammar files)
        if (suite->specialsBefore.length() > 0) {
            body[0]->specialsBefore.append(suite->specialsBefore);
        }

        if (suite->specialsAfter.length() > 0) {
            body[body.length() - 1]->specialsAfter.append(suite->specialsAfter);
        }
    }
    return body;
}

SimpleNodePtr PythonTreeBuilder::handleIfConstruct(const SimpleNodePtr& n, int arity)
{
    StmtTypeList body;
    exprTypePtr test;

    suiteTypePtr orelse;
    if (arity % 3 == 1) {
        arity -= 2;
        orelse = popSuiteAndSuiteType();
    }

    //make the suite
    SimpleNodePtr node = _stack->popNode();
    SuitePtr suite = node.dynamicCast<Suite>();//_stack->popNode<Suite>();
    arity--;
    body = suite->body();
    test = _stack->popNode<exprType>();
    arity--;

    //make the if
    IfPtr last;
    if (arity == 0) {
        //last If found
        last = n.dynamicCast<If>();
    } else {
        last = _stack->popNode<If>();
        arity--;
    }
    last->setTest(test);
    last->setBody(body);
    last->setOrelse(orelse);
    addSpecialsAndClearOriginal(suite, last);

    while (arity > 0) {
        suite = _stack->popNode<Suite>();
        arity--;

        body = suite->body();
        test = _stack->popNode<exprType>();
        arity--;

        suiteTypePtr newOrElse = SuitePtr(new Suite(StmtTypeList(last)));
        if (arity == 0) {
            //last If found
            last = n.dynamicCast<If>();
        } else {
            last = _stack->popNode<If>();
            arity--;
        }
        last->setTest(test);
        last->setBody(body);
        last->setOrelse(newOrElse);
        addSpecialsAndClearOriginal(suite, last);
    }
    return last;
}

SimpleNodePtr PythonTreeBuilder::makeDecorator(const QList<SimpleNodePtr>& nodes)
{
    exprTypePtr starargs;
    exprTypePtr kwargs;
    exprTypePtr func;

    QList<SimpleNodePtr> keywordsl;
    QList<SimpleNodePtr> argsl;

    QListIterator<SimpleNodePtr> iter(nodes);
    while (iter.hasNext()) {
        SimpleNodePtr node = iter.next();
        if (node->nodeId() == JJTEXTRAKEYWORDVALUELIST) {
            ExtraArgValuePtr extraArg = node.dynamicCast<ExtraArgValue>();
            kwargs = extraArg->value;
            addSpecialsAndClearOriginal(extraArg, kwargs);
            extraArg->specialsBefore = kwargs->specialsBefore;
            extraArg->specialsAfter = kwargs->specialsAfter;
        } else if (node->nodeId() == JJTEXTRAARGVALUELIST) {
            ExtraArgValuePtr extraArg = node.dynamicCast<ExtraArgValue>();
            starargs = extraArg->value;
            addSpecialsAndClearOriginal(extraArg, starargs);
            extraArg->specialsBefore = starargs->specialsBefore;
            extraArg->specialsAfter = starargs->specialsAfter;
        } else if (node.dynamicCast<keywordType>()) {
            keywordTypePtr keyword = node.dynamicCast<keywordType>();
            if(!starargs){
                keyword->setAfterstarargs(true);
            }
            keywordsl.append(keyword);
        } else if (isArg(node)){
            //default
            argsl.append(node);
        } else if (node.dynamicCast<Comprehension>()) {
            argsl.append(ListCompPtr(new ListComp(iter.next().dynamicCast<exprType>(), ComprehensionTypeList(node.dynamicCast<comprehensionType>()), CompContextType::EmptyCtx)));
        } else if (node.dynamicCast<ComprehensionCollection>()) {
            //list comp (2 nodes: comp type and the elt -- what does elt mean by the way?)
            argsl.append(ListCompPtr(new ListComp(iter.next().dynamicCast<exprType>(), node.dynamicCast<ComprehensionCollection>()->getGenerators(), CompContextType::EmptyCtx)));
        } else if (node.dynamicCast<decoratorsType>()) {
            func = _stack->popNode<exprType>();//the func is the last thing in the stack
            decoratorsTypePtr d = node.dynamicCast<decoratorsType>();
            d->setFunc(func);
            d->setArgs(ExprTypeList());
            d->setKeywords(KeywordTypeList());
            d->setStarargs(starargs);
            d->setKwargs(kwargs);
            return d;
        } else {
            argsl.append(node);
        }

    }
    throw QString("Something wrong happened while making the decorators...");
}

bool PythonTreeBuilder::isArg(const SimpleNodePtr& n)
{
    return n.dynamicCast<ExtraArg>() || n.dynamicCast<DefaultArg>() || n.dynamicCast<keywordType>();
}

ExprTypeList PythonTreeBuilder::makeExprs()
{
    SimpleNodePtr commaNode;
    if (_stack->nodeArity() > 0 && _stack->peekNode()->nodeId() == JJTCOMMA){
        commaNode = _stack->popNode();
    }
    int arity = _stack->nodeArity();
    ExprTypeList exprs = makeExprs(arity);
    if(commaNode && arity > 0){
        exprs[arity-1]->addSpecial(SpecialStrPtr(new SpecialStr(",", commaNode->beginLine, commaNode->beginColumn)), true);
    }
    return exprs;
}

ExprTypeList PythonTreeBuilder::makeExprs(int l)
{
    ExprTypeList exprs;
    for (int i = l - 1; i >= 0; i--) {
        _lastPop = _stack->popNode();
        exprs.insert(0, _lastPop.dynamicCast<exprType>());
    }
    return exprs;
}

AliasTypeList PythonTreeBuilder::makeAliases(int l)
{
    AliasTypeList aliases;
    for (int i = l - 1; i >= 0; i--) {
        aliases.append(_stack->popNode<aliasType>());
    }
    return aliases;
}

NameTokPtr PythonTreeBuilder::makeName(int _ctx)
{
    NamePtr name = _stack->popNode<Name>();
    if (name)
        return makeName(_ctx, name);
    return NameTokPtr(NULL);
}

NameTokPtr PythonTreeBuilder::makeName(int _ctx, const NamePtr& name)
{
    NameTokPtr n = NameTokPtr(new NameTok(name->id(), _ctx));
    n->beginColumn = name->beginColumn;
    n->beginLine = name->beginLine;
    addSpecials(name, n);
    //we have to create it because it could be that specials are added later
    //(so, the instance must be already created even if not used)
    name->specialsBefore = n->getSpecialsAfter();
    name->specialsAfter = n->getSpecialsBefore();
    return n;
}

argumentsTypePtr PythonTreeBuilder::makeArguments(int l)
{
    NameTokPtr kwarg;
    NameTokPtr stararg;
    exprTypePtr varargannotation;
    exprTypePtr kwargannotation;

    QList<SimpleNodePtr> list;
    for (int i = l-1; i >= 0; i--) {
        SimpleNodePtr popped = _stack->popNode();
        if (popped->nodeId() == JJTEXTRAKEYWORDLIST) {
            ExtraArgPtr node = popped.dynamicCast<ExtraArg>();
            kwarg = node->tok;
            kwargannotation = node->typeDef;
            addSpecialsAndClearOriginal(node, kwarg);
        } else if (popped->nodeId() == JJTEXTRAARGLIST) {
            ExtraArgPtr node = popped.dynamicCast<ExtraArg>();
            stararg = node->tok;
            varargannotation = node->typeDef;
            if(stararg){
                //can happen, as in 3.0 we can have a single '*'
                addSpecialsAndClearOriginal(node, stararg);
            }
        } else {
            list.append(popped);
        }
    }
    QList<DefaultArgPtr> rlist;
    for(int i = list.length()-1; i >= 0; --i){
        rlist.append(list[i].dynamicCast<DefaultArg>());
    }
    argumentsTypePtr arguments = __makeArguments(rlist, stararg, kwarg);
    arguments->setVarargannotation(varargannotation);
    arguments->setKwargannotation(kwargannotation);
    return arguments;
}

argumentsTypePtr PythonTreeBuilder::__makeArguments(const QList<DefaultArgPtr>& def, const NameTokPtr& varg, const NameTokPtr& kwarg)
{
    ExprTypeList fpargs;
    ExprTypeList fpargsAnn;
    ExprTypeList fpargsDefaults;


    ExprTypeList kwonlyargs;
    ExprTypeList kwonlyargsAnn;
    ExprTypeList kwonlyargsDefaults;

    for(int i = 0 ; i< def.length(); i++){
        DefaultArgPtr node = def[i];
        exprTypePtr parameter = node->parameter;

        if (node->nodeId() == JJTONLYKEYWORDARG || node->nodeId() == JJTONLYKEYWORDARG2) {
            _ctx.setKwOnlyParam(parameter);
            kwonlyargs.append(parameter);
            kwonlyargsAnn.append(node->typeDef);
            kwonlyargsDefaults.append(node->value);
        } else {
            //regular parameter
            _ctx.setParam(parameter);
            fpargs.append(parameter);
            fpargsAnn.append(node->typeDef);
            fpargsDefaults.append(node->value);
        }

        if(node->specialsBefore.length() > 0){
            parameter->specialsBefore.append(node->specialsBefore);
        }
        if(node->specialsAfter.length() > 0){
            parameter->specialsAfter.append(node->specialsAfter);
        }
    }
    return argumentsTypePtr(new argumentsType(
        fpargs,
        varg,
        kwarg,
        fpargsDefaults,

        //new on Python 3.0
        kwonlyargs,
        kwonlyargsDefaults,

        //annotations
        fpargsAnn,
        exprTypePtr(),  //this one will be set later on makeArguments (varargannotation)
        exprTypePtr(),  //this one will be set later on makeArguments (kwargannotation)
        kwonlyargsAnn
    ));
}

void PythonTreeBuilder::setParentForFuncOrClass(const StmtTypeList& body, const stmtTypePtr& classDef)
{
    foreach(stmtTypePtr b, body) {
        if (b.dynamicCast<ClassDef>()) {
            b.dynamicCast<ClassDef>()->parent = classDef;
        } else if(b.dynamicCast<FunctionDef>()) {
            b.dynamicCast<FunctionDef>()->parent = classDef.toWeakRef();
        }
    }
}

NameTokList PythonTreeBuilder::makeIdentifiers(int _ctx)
{
    int l = _stack->nodeArity();
    return makeIdentifiers(_ctx, l);
}

NameTokList PythonTreeBuilder::makeIdentifiers(int _ctx, int arity)
{
    NameTokList ids;
    for (int i = arity - 1; i >= 0; i--) {
        ids.append(makeName(_ctx));
    }
    return ids;
}

SimpleNodePtr PythonTreeBuilder::makeWithStmt(int arity)
{
    SuitePtr suite = _stack->popNode<Suite>();
    arity--;

    QVector<WithItemTypePtr> items;
    items.resize(arity);
    while (arity>0) {
        items[arity-1] = _stack->popNode<WithItemType>();
        arity--;
    }


    suiteTypePtr s = SuitePtr(new Suite(suite->body()));
    addSpecialsAndClearOriginal(suite, s);

    return WithPtr(new With(items.toList(), s));
}

SimpleNodePtr PythonTreeBuilder::makeWithItem(int arity)
{
    exprTypePtr expr = _stack->popNode<exprType>(); //expr
    arity--;

    exprTypePtr asExpr;
    if (arity > 0) {
        asExpr = expr;
        expr = _stack->popNode<exprType>();
        _ctx.setStore(asExpr);
    }
    return WithItemPtr(new WithItem(expr, asExpr));
}

SimpleNodePtr PythonTreeBuilder::makeTuple(const SimpleNodePtr& n)
{
    //try {
    bool endsWithComma = false;
    //There's a detail with tuples: if we have a tuple with a single element and it doesn't end with a comma,
    //it's not actually treated as a tuple, but as its only item.
    if (_stack->nodeArity() > 0 && _stack->peekNode()->nodeId() == JJTCOMMA){
        endsWithComma = true;
    }

    if(!endsWithComma && _stack->nodeArity() == 1){
        QList<ObjectPtr> tokenSourceSpecialTokensList = *_stack->grammar()->getTokenSourceSpecialTokensList();
        foreach(ObjectPtr object, tokenSourceSpecialTokensList) {
            /*if (dynamic_cast<QList<Object*> >(object)) {
                QList<Object*> objects = dynamic_cast<QList<Object*> >(object);
                object = objects[0];
            }*/
            if (object.dynamicCast<SpecialStr>()) {
                SpecialStrPtr specialStr = object.dynamicCast<SpecialStr>();
                if( specialStr->str.str() == ",") {
                    endsWithComma = true;
                    break;
                }
            }
        }
    }

    ExprTypeList exp = makeExprs();
    TuplePtr t = TuplePtr(new Tuple(exp, ExprContextType::Load, endsWithComma));
    addSpecialsAndClearOriginal(n, t);
    return t;
    /*} catch (ClassCastException e) {
        if(e.getMessage().equals(ExtraArgValue.class.getName())){
            this.stack.getGrammar().addAndReport(
                    new ParseException("Token: '*' is not expected inside tuples.", lastPop),
                    "Treated class cast exception on tuple");
        }
        this.stack.getGrammar().addAndReport(
                new ParseException("Syntax error while detecting tuple.", lastPop),
                "Treated class cast exception on tuple");

        while(stack.nodeArity() > 0){
            //clear whatever we had in this construct...
            stack.popNode();
        }

        //recover properly!
        return new Tuple(new exprType[0], Tuple.Load, false);

    }*/
}

SimpleNodePtr PythonTreeBuilder::makeDictionaryOrSet(int arity)
{
    if(arity == 0){
        return DictPtr(new Dict(ExprTypeList(), ExprTypeList()));
    }

    SimpleNodePtr dictNode0 = _stack->popNode();

    if (dictNode0.dynamicCast<Set>()) {
        SetPtr set = dictNode0.dynamicCast<Set>();
        ExprTypeList elts;
        for (int i = arity-2; i >= 0; i--) {
            elts.append(_stack->popNode<exprType>());
        }
        set->setElts(elts);
        return set;
    }


    if (dictNode0.dynamicCast<ComprehensionCollection>()) {
        if(arity == 2){
            ComprehensionCollectionPtr comp = dictNode0.dynamicCast<ComprehensionCollection>();
            return SetCompPtr(new SetComp(_stack->popNode<exprType>(), comp->getGenerators()));

        } else if(arity == 3) {
            SimpleNodePtr dictNode1 = _stack->popNode(); //we must inverse things here...
            ComprehensionCollectionPtr comp = dictNode0.dynamicCast<ComprehensionCollection>();
            return DictCompPtr(new DictComp(_stack->popNode<exprType>(), dictNode1.dynamicCast<exprType>(), comp->getGenerators()));
        }
    }

    bool isDictComplete = arity % 2 == 0;
    int l = arity / 2;
    ExprTypeList keys;

    /*if (isDictComplete) {
        keys = new exprType[l];
    }else{
        keys = new exprType[l+1]; //we have 1 additional entry in the keys (parse error actually, but let's recover at this point!)
    }*/
    bool node0Used = false;
    ExprTypeList vals;
    for (int i = l - 1; i >= 0; i--) {
        if (!node0Used) {
            node0Used = true;
            vals.append(dictNode0.dynamicCast<exprType>());
            keys.append(_stack->popNode<exprType>());
        } else {
            vals.append(_stack->popNode<exprType>());
            keys.append(_stack->popNode<exprType>());
        }
    }

    if(!isDictComplete){
        if (node0Used) {
            keys.append(_stack->popNode<exprType>());
        } else {
            keys.append(dictNode0.dynamicCast<exprType>());
        }
    }

    return DictPtr(new Dict(keys, vals));
}

ComprehensionCollectionPtr PythonTreeBuilder::makeCompFor(int arity)
{
    ComprehensionCollectionPtr col;
    if (_stack->peekNode<ComprehensionCollection>()) {
        col = _stack->popNode<ComprehensionCollection>();
        arity--;
    } else {
        col = ComprehensionCollectionPtr(new ComprehensionCollection());
    }

    ExprTypeList ifs;
    for (int i = arity-3; i >= 0; i--) {
        ifs.append(_stack->popNode<exprType>());
    }
    exprTypePtr iter = _stack->popNode<exprType>();
    exprTypePtr target = _stack->popNode<exprType>();
    _ctx.setStore(target);
    col->added.append(ComprehensionPtr(new Comprehension(target, iter, ifs)));
    return col;
}

SimpleNodePtr PythonTreeBuilder::makeImportFrom25Onwards(int arity)
{
    AliasTypeList aliastL;
    while(arity > 0 && _stack->peekNode<aliasType>()) {
        aliastL.prepend(_stack->popNode<aliasType>());
        arity--;
    }

    NameTokPtr nT;
    if (arity > 0) {
        nT = makeName(NameContextType::ImportModule);
    } else {
        nT = NameTokPtr(new NameTok("", NameContextType::ImportModule));
        ObjectPtr temporaryTok = _stack->grammar()->temporaryToken;
        SpecialStrPtr temporaryToken;

        if (temporaryTok.dynamicCast<SpecialStr>()) {
            temporaryToken = temporaryTok.dynamicCast<SpecialStr>();
        } else {
            //must be a Token
            temporaryToken = temporaryTok.dynamicCast<Token>()->asSpecialStr();
        }

        if (temporaryToken->str.str() == "from") {
            nT->beginColumn = temporaryToken->beginCol;
            nT->beginLine = temporaryToken->beginLine;
        } else {
            //Log.log("Expected to find 'from' token as the current temporary token (begin col/line can be wrong)!");
        }
    }
    return ImportFromPtr(new ImportFrom(nT, aliastL, 0));
}
